import com.google.common.base.Joiner;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;

/**
 * Created by donar on 17/3/13.
 */
public class DXMSpider {
    static JdbcUtil jdbcUtil = new JdbcUtil();
    static LinkedBlockingQueue<String> idpaths = new LinkedBlockingQueue<String>();
    public static void main(String[] args) throws SQLException, IOException {
        //fetchCatalogFilesRelation();
        fetchProductTypeFiles();
    }

    private static void fetchCatalogFilesRelation() throws SQLException {
        Connection connection = jdbcUtil.getConnection();
        PreparedStatement pstmt = connection.prepareStatement("select id_path,site from amazon_catagory where has_children=0 and product_type_files is  null");
        ResultSet rs = pstmt.executeQuery();
        while(rs.next()){
            String idpath=rs.getString("id_path");
            String site=rs.getString("site");
            if(site.equals("USA"))site="us";
            if(site.equals("CANADA"))site="ca";
            if(site.equals("MEXICO"))site="mx";
            idpaths.offer(idpath+";"+site);
        }
        rs.close();
        pstmt.close();
        JdbcUtil.releaseConn(connection);
        Executor executor = Executors.newFixedThreadPool(100);
        for (int i = 0; i < 100; i++) {
            executor.execute(new FetchDXMTask(idpaths));
        }
    }

    private static void fetchProductTypeFiles() throws SQLException, IOException {
        CloseableHttpClient closeableHttpClient = HttpClientBuilder.create().build();
        Connection connection = JdbcUtil.getConnection();
        PreparedStatement preparedStatement = connection.prepareStatement("select  product_type_files from amazon_catagory where product_type_files is not null ");
        ResultSet rs = preparedStatement.executeQuery();
        Set<String> fileSet = new HashSet<String>();
        while (rs.next()){
            String productFiles = rs.getString("product_type_files");
            String[] files = productFiles.split(",");
            fileSet.addAll(Arrays.asList(files));
        }
        rs.close();
        preparedStatement.close();
        for(String file : fileSet){
            if ("".equals(file))continue;
            String url = "http://rs.beta.dianxiaomi.com/amazon/definitions/na/product/"+file+"?v=5.6";
            System.out.println(url);
            HttpGet httpGet = new HttpGet(url);
            FetchDXMTask.addHeader(httpGet);
            HttpResponse httpResponse = closeableHttpClient.execute(httpGet);
            String content = EntityUtils.toString(httpResponse.getEntity());
            PreparedStatement pstmt = connection.prepareStatement("replace into amazon_json_schema (`name`,`schema`) VALUES (?,?)");
            pstmt.setString(1,file);
            pstmt.setString(2,content);
            pstmt.execute();
            pstmt.close();
        }
    }
}
